{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adversarial-Robustness-Toolbox for scikit-learn LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "from art.classifiers import SklearnClassifier\n",
    "from art.attacks import ProjectedGradientDescent\n",
    "from art.utils import load_mnist\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load and transform MNIST dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test), min_, max_ = load_mnist()\n",
    "\n",
    "n_samples_train = x_train.shape[0]\n",
    "n_features_train = x_train.shape[1] * x_train.shape[2] * x_train.shape[3]\n",
    "n_samples_test = x_test.shape[0]\n",
    "n_features_test = x_test.shape[1] * x_test.shape[2] * x_test.shape[3]\n",
    "\n",
    "x_train = x_train.reshape(n_samples_train, n_features_train)\n",
    "x_test = x_test.reshape(n_samples_test, n_features_test)\n",
    "\n",
    "y_train = np.argmax(y_train, axis=1)\n",
    "y_test = np.argmax(y_test, axis=1)\n",
    "\n",
    "n_samples_max = 100\n",
    "x_train = x_train[0:n_samples_max]\n",
    "y_train = y_train[0:n_samples_max]\n",
    "x_test = x_test[0:n_samples_max]\n",
    "y_test = y_test[0:n_samples_max]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train LogisticRegression classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LogisticRegression(penalty='l2', dual=False, tol=0.0001, C=1.0, fit_intercept=True, intercept_scaling=1, \n",
    "                           class_weight='balanced', random_state=None, solver='lbfgs', max_iter=100, \n",
    "                           multi_class='ovr', verbose=0, warm_start=False, n_jobs=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight='balanced', dual=False,\n",
       "                   fit_intercept=True, intercept_scaling=1, l1_ratio=None,\n",
       "                   max_iter=100, multi_class='ovr', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X=x_train, y=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create and apply ProjectedGradientDescent Attack with ART"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "art_classifier = SklearnClassifier(model=model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pgd = ProjectedGradientDescent(classifier=art_classifier, norm=np.inf, eps=.3, eps_step=0.1, max_iter=20, \n",
    "                               targeted=False, num_random_init=0, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "x_train_adv = pgd.generate(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_adv = pgd.generate(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate LogisticRegression classifier on benign and adversarial samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Benign Training Score: 1.0000\n"
     ]
    }
   ],
   "source": [
    "score = model.score(x_train, y_train)\n",
    "print(\"Benign Training Score: %.4f\" % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAECCAYAAAD+eGJTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAO/0lEQVR4nO3df2xd9X3G8edpYpIFQhsvJUtZCmlIBy2sobP4ISKgQmVZNQnQVFhUVSnrFtaStmyZBIumwSY6ZRPQUcqQwsgIEtBCgZE/WFsUIaAaeJiMQiAFWggbwTgECwKUhsT+7A+fbB61v3Z8r++59uf9kiJfn+f6ng8n8HDuPV/f64gQgLw+UPcAAOpFCQDJUQJAcpQAkBwlACRHCQDJ1VICtlfYftb2z2xfVscMJbZ32H7K9hO2e9pgno22d9neNmxbp+37bT9ffZ3XZvNdYXtndQyfsP25GudbZPsB28/Yftr2N6rtbXEMC/O15Bi61esEbM+Q9Jykz0p6WdJjklZGxDMtHaTA9g5JXRGxu+5ZJMn26ZLelnRLRBxfbfsHSf0Rsb4q0nkRcWkbzXeFpLcj4qo6ZhrO9kJJCyNiq+25kh6XdK6kL6kNjmFhvvPVgmNYx5nASZJ+FhEvRMR7kr4r6Zwa5pgyIuIhSf3v23yOpE3V7U0a+pemFqPM1zYiojcitla335K0XdKRapNjWJivJeoogSMl/few719WC/+Bxykk/cj247ZX1z3MKBZERG91+1VJC+ocZhRrbD9ZPV2o7enKcLaPlnSipG614TF833xSC44hLwyObHlEfFrS70m6uDrdbVsx9Jyu3dZ/3yBpiaRlknolXV3vOJLtwyTdJemSiNgzPGuHYzjCfC05hnWUwE5Ji4Z9/5vVtrYRETurr7sk3aOhpzDtpq96LnngOeWumuf5fyKiLyIGImJQ0o2q+Rja7tDQf2C3RsTd1ea2OYYjzdeqY1hHCTwmaantxbYPkfSHkjbXMMeIbB9avTgj24dKOlvStvJP1WKzpFXV7VWS7q1xll9x4D+uynmq8RjatqSbJG2PiGuGRW1xDEebr1XHsOVXBySputTxj5JmSNoYEd9s+RCjsP0xDf3fX5JmSrqt7vls3y7pTEnzJfVJulzSv0q6Q9JHJb0k6fyIqOXFuVHmO1NDp7EhaYeki4Y9/271fMslPSzpKUmD1eZ1GnreXfsxLMy3Ui04hrWUAID2wQuDQHKUAJAcJQAkRwkAyVECQHK1lkAbL8mVxHyNauf52nk2qbXz1X0m0NZ/EWK+RrXzfO08m9TC+eouAQA1a2ixkO0Vkq7V0Mq/f46I9aX7H+JZMVuH/u/3+7RXHZo14f1PNuZrTDvP186zSc2f75d6R+/FXo+UTbgEJvLmIIe7M072WRPaH4CJ644t2hP9I5ZAI08HeHMQYBpopASmwpuDABjDzMneQXWpY7Ukzdacyd4dgIPUyJnAuN4cJCI2RERXRHS18wsxQFaNlEBbvzkIgPGZ8NOBiNhve42kH+r/3hzk6aZNBqAlGnpNICLuk3Rfk2YBUANWDALJUQJAcpQAkBwlACRHCQDJUQJAcpQAkBwlACRHCQDJUQJAcpQAkBwlACRHCQDJUQJAcpQAkBwlACRHCQDJUQJAcpQAkBwlACRHCQDJUQJAcpQAkBwlACRHCQDJUQJAcpQAkBwlACRHCQDJUQJAcg19NDmmFs8s/3XP+PD8Sd3/s39xdDEfmDNYzI9asquYz/mqi/mr1xxSzLd2fa+Y7x54p5iffOfaYn7Mnz9azOvSUAnY3iHpLUkDkvZHRFczhgLQOs04E/hMROxuwuMAqAGvCQDJNVoCIelHth+3vboZAwForUafDiyPiJ22j5B0v+2fRsRDw+9QlcNqSZqtOQ3uDkCzNXQmEBE7q6+7JN0j6aQR7rMhIroioqtDsxrZHYBJMOESsH2o7bkHbks6W9K2Zg0GoDUaeTqwQNI9tg88zm0R8YOmTDVNzThuaTGPWR3F/JUzPlTM3z2lfB2784Pl/OFPla+T1+3ffjG3mP/9d1YU8+4TbivmL+57t5iv7/tsMf/Iw1HM29WESyAiXpD0qSbOAqAGXCIEkqMEgOQoASA5SgBIjhIAkqMEgOR4P4EmGjjz08X8mpuvL+Yf7yj/vvt0ty8GivlfX/elYj7znfJ1+lPvXFPM5+7cX8xn7S6vI5jT013M2xVnAkBylACQHCUAJEcJAMlRAkBylACQHCUAJMc6gSaa9ewrxfzxXy4q5h/v6GvmOE23tveUYv7C2+XPLbh5yfeL+ZuD5ev8C77978V8sk3NdwsYG2cCQHKUAJAcJQAkRwkAyVECQHKUAJAcJQAk54jWXf083J1xss9q2f7aTf+FpxbzPSvKnwsw48nDivlPvnrdQc803JW7f7uYP3ZGeR3AwBtvFvM4tfwO9Tu+Xoy1eOVPynfAqLpji/ZEv0fKOBMAkqMEgOQoASA5SgBIjhIAkqMEgOQoASA51gm0kRnzf72YD7zeX8xfvK18nf/p0zcW85P+7mvF/Ijr6/19fkxcQ+sEbG+0vcv2tmHbOm3fb/v56uu8Zg4MoHXG83TgZkkr3rftMklbImKppC3V9wCmoDFLICIekvT+89BzJG2qbm+SdG6T5wLQIhN9YXBBRPRWt1+VtKBJ8wBosYavDsTQK4ujvrpoe7XtHts9+7S30d0BaLKJlkCf7YWSVH3dNdodI2JDRHRFRFeHZk1wdwAmy0RLYLOkVdXtVZLubc44AFptzM8dsH27pDMlzbf9sqTLJa2XdIftL0t6SdL5kzlkFgO7X2/o5/ftOaShn//kF54p5q/dMKP8AIMDDe0f9RizBCJi5SgRq36AaYBlw0BylACQHCUAJEcJAMlRAkBylACQ3JiXCDF1HHfpc8X8whPKV3X/5agtxfyMz19czOd+79FijvbEmQCQHCUAJEcJAMlRAkBylACQHCUAJEcJAMmxTmAaGXjjzWL++leOK+b/tfndYn7ZlbcU8788/7xiHv/5wWK+6JuPFHO18DMyMuFMAEiOEgCSowSA5CgBIDlKAEiOEgCSowSA5BwtvPZ6uDvjZPNO5e2q/49OLea3Xn5VMV88c3ZD+//kLWuK+dIbe4v5/hd2NLT/6aw7tmhP9HukjDMBIDlKAEiOEgCSowSA5CgBIDlKAEiOEgCSY50Axi1OW1bMD1//cjG//WM/bGj/xz7wx8X8t/6m/H4KA8+/0ND+p7KG1gnY3mh7l+1tw7ZdYXun7SeqP59r5sAAWmc8TwdulrRihO3fiohl1Z/7mjsWgFYZswQi4iFJ/S2YBUANGnlhcI3tJ6unC/OaNhGAlppoCdwgaYmkZZJ6JV092h1tr7bdY7tnn/ZOcHcAJsuESiAi+iJiICIGJd0o6aTCfTdERFdEdHVo1kTnBDBJJlQCthcO+/Y8SdtGuy+A9jbmOgHbt0s6U9J8SX2SLq++XyYpJO2QdFFElH/ZW6wTmO5mLDiimL9ywTHFvPvSa4v5B8b4f9YXXjy7mL+5/PViPp2V1gmM+eEjEbFyhM03NTwVgLbAsmEgOUoASI4SAJKjBIDkKAEgOUoASI73E0DbuOPlR4r5HB9SzH8R7xXz3//aJeXHv6e7mE9lfO4AgFFRAkBylACQHCUAJEcJAMlRAkBylACQ3Ji/SgwcMLi8/LkDP//87GJ+/LIdxXysdQBjua7/xPLj39vT0ONPV5wJAMlRAkBylACQHCUAJEcJAMlRAkBylACQHOsEEnHX8cX8ua+Xr9PfeNqmYn767PLv8zdqb+wr5o/2Ly4/wOCYH42REmcCQHKUAJAcJQAkRwkAyVECQHKUAJAcJQAkxzqBKWTm4qOK+c8v/Egxv+KC7xbzPzhs90HP1Ezr+rqK+YPXnlLM520qf24BRjbmmYDtRbYfsP2M7adtf6Pa3mn7ftvPV1/nTf64AJptPE8H9ktaGxGfkHSKpIttf0LSZZK2RMRSSVuq7wFMMWOWQET0RsTW6vZbkrZLOlLSOZIOrCPdJOncyRoSwOQ5qBcGbR8t6URJ3ZIWRMSBxdivSlrQ1MkAtMS4S8D2YZLuknRJROwZnsXQp5qO+Mmmtlfb7rHds097GxoWQPONqwRsd2ioAG6NiLurzX22F1b5Qkm7RvrZiNgQEV0R0dWhWc2YGUATjefqgCXdJGl7RFwzLNosaVV1e5Wke5s/HoDJNp51AqdJ+qKkp2w/UW1bJ2m9pDtsf1nSS5LOn5wRp4+ZR3+0mL/5OwuL+QV/+4Ni/qcfuruYT7a1veXr+I/8U3kdQOfN/1HM5w2yDmAyjFkCEfFjSR4lPqu54wBoNZYNA8lRAkBylACQHCUAJEcJAMlRAkByvJ/AQZi58DeKef/GQ4v5VxY/WMxXzu076Jmaac3O5cV86w3Livn8728r5p1vcZ2/HXEmACRHCQDJUQJAcpQAkBwlACRHCQDJUQJAcqnWCbz3u+XfZ3/vz/qL+bpj7ivmZ//aOwc9UzP1DbxbzE/fvLaYH/tXPy3mnW+Ur/MPFlO0K84EgOQoASA5SgBIjhIAkqMEgOQoASA5SgBILtU6gR3nljvvuRPunNT9X//GkmJ+7YNnF3MPjPbO70OOvfLFYr60r7uYDxRTTFecCQDJUQJAcpQAkBwlACRHCQDJUQJAcpQAkJwjonwHe5GkWyQtkBSSNkTEtbavkPQnkl6r7rouIoq/cH+4O+Nk82nmQKt1xxbtif4RF5qMZ7HQfklrI2Kr7bmSHrd9f5V9KyKuatagAFpvzBKIiF5JvdXtt2xvl3TkZA8GoDUO6jUB20dLOlHSgfWna2w/aXuj7XlNng1AC4y7BGwfJukuSZdExB5JN0haImmZhs4Urh7l51bb7rHds097mzAygGYaVwnY7tBQAdwaEXdLUkT0RcRARAxKulHSSSP9bERsiIiuiOjq0KxmzQ2gScYsAduWdJOk7RFxzbDtC4fd7TxJ5Y+kBdCWxnN14DRJX5T0lO0nqm3rJK20vUxDlw13SLpoUiYEMKnGc3Xgx5JGur5YfhN+AFMCKwaB5CgBIDlKAEiOEgCSowSA5CgBIDlKAEiOEgCSowSA5CgBIDlKAEiOEgCSowSA5CgBIDlKAEhuzM8daOrO7NckvTRs03xJu1s2wMFjvsa083ztPJvU/PmOiogPjxS0tAR+Zed2T0R01TbAGJivMe08XzvPJrV2Pp4OAMlRAkBydZfAhpr3Pxbma0w7z9fOs0ktnK/W1wQA1K/uMwEANaMEgOQoASA5SgBIjhIAkvsfsRZSmOVUgvYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(x_train[0, :].reshape((28, 28)))\n",
    "plt.clim(0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Benign Training Predicted Label: 5\n"
     ]
    }
   ],
   "source": [
    "prediction = model.predict(x_train[0:1, :])[0]\n",
    "print(\"Benign Training Predicted Label: %i\" % prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adversarial Training Score: 0.2400\n"
     ]
    }
   ],
   "source": [
    "score = model.score(x_train_adv, y_train)\n",
    "print(\"Adversarial Training Score: %.4f\" % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAECCAYAAAD+eGJTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAQrklEQVR4nO3dfYwd5XXH8d8JXuxgG/ALrFwDARNHgaSKIctLAEUUq9ShCS9qQfUf1JVQjdJQQZO2IFIJKpGWVIGUf0JlAoppCRUIKK7q8lKDaiwShzV1sbGpDcYQG3sXMMa8ml379I8dp7dk93l2d+7cGXO+Hwnt3Tn37hzG65/n3ueZZ8zdBSCuT9XdAIB6EQJAcIQAEBwhAARHCADBEQJAcLWEgJktMLP/MbMXzey6OnpIMbOtZrbOzNaaWW8D+rnLzPrNbH3Ltulm9riZbS6+TmtYfzea2fbiGK41swtq7O9YM3vSzDaY2fNmdnWxvRHHMNFfR46hdXqegJkdImmTpN+WtE3SM5IWuvuGjjaSYGZbJfW4+xt19yJJZvZVSe9Kutvdv1hs+ztJu9z95iJIp7n7tQ3q70ZJ77r7D+roqZWZzZI0y92fNbOpktZIuljSH6kBxzDR32XqwDGs40zgdEkvuvsWd/9I0j9LuqiGPg4a7r5S0q6Pbb5I0tLi8VIN/dLUYoT+GsPdd7j7s8XjdyRtlDRbDTmGif46oo4QmC3ply3fb1MH/4dHySU9ZmZrzGxx3c2MoNvddxSPd0rqrrOZEVxlZs8Vbxdqe7vSysyOl3SKpNVq4DH8WH9SB44hHwwO7xx3P1XS1yR9qzjdbSwfek/XtPnft0s6UdI8STsk3VJvO5KZTZH0gKRr3H1Pa60Jx3CY/jpyDOsIge2Sjm35/phiW2O4+/bia7+khzT0FqZp+or3kgfeU/bX3M//4+597r7P3fdLukM1H0Mz69LQX7B73P3BYnNjjuFw/XXqGNYRAs9ImmtmJ5jZoZL+QNKyGvoYlplNLj6ckZlNlnS+pPXpV9VimaRFxeNFkh6usZdfc+AvV+ES1XgMzcwk3Slpo7vf2lJqxDEcqb9OHcOOjw5IUjHU8feSDpF0l7t/r+NNjMDM5mjoX39JmiDpp3X3Z2b3SjpX0kxJfZJukPQvku6TdJykVyRd5u61fDg3Qn/naug01iVtlXRly/vvTvd3jqSnJK2TtL/YfL2G3nfXfgwT/S1UB45hLSEAoDn4YBAIjhAAgiMEgOAIASA4QgAIrtYQaPCUXEn0V1aT+2tyb1Jn+6v7TKDRfxCiv7Ka3F+Te5M62F/dIQCgZqUmC5nZAkm3aWjm34/d/ebU8w+1iT5Jk3/1/YD2qksTx73/qtFfOU3ur8m9Se3v70O9p498rw1XG3cIjGdxkMNtup9h88e1PwDjt9pXaI/vGjYEyrwdYHEQ4BOgTAgcDIuDAMiYUPUOiqGOxZI0SYdVvTsAY1TmTGBUi4O4+xJ373H3niZ/EANEVSYEGr04CIDRGffbAXcfNLOrJD2q/1sc5Pm2dQagI0p9JuDuyyUtb1MvAGrAjEEgOEIACI4QAIIjBIDgCAEgOEIACK7yacOIY/C8L9e6/wlPrKl1/wcrzgSA4AgBIDhCAAiOEACCIwSA4AgBIDhCAAiOeQL4ld2Xf6XU66ds/6hNnYxPbp4C8wiGx5kAEBwhAARHCADBEQJAcIQAEBwhAARHCADBMU8gkAlzjk/Wc+P8uXH2utcTyMn19+7sQyvd/5H/+LNKf/54cSYABEcIAMERAkBwhAAQHCEABEcIAMERAkBwzBNokKqv5x+YYMn6myelx8mnzDxzzD21mrT4tWR9xqT3kvVnf/a59OvXpfe/e266fuk3ViXruwYmJ+ub/vLk9A4aqlQImNlWSe9I2idp0N172tEUgM5px5nAb7n7G234OQBqwGcCQHBlQ8AlPWZma8xscTsaAtBZZd8OnOPu283saEmPm9kL7r6y9QlFOCyWpEk6rOTuALRbqTMBd99efO2X9JCk04d5zhJ373H3ni5NLLM7ABUYdwiY2WQzm3rgsaTzJa1vV2MAOqPM24FuSQ+Z2YGf81N3f6QtXY1T7nr5N86elaznrveuel373P53Xn1Wsv7e7PSZ1pSv9ifrC2a9mKx/v3ttsl7WgO9L1v+9O318vv+bC5L1i7u3JOuvfjAtWX9qY3qewknPpX9++v+uPuMOAXffIulLbewFQA0YIgSCIwSA4AgBIDhCAAiOEACCIwSA4D5R6wkMbtmafkJmnkDuev7c9fo5R6yakayfMPnNzE94OlntffO4ZL1nxqvJem4ewLV985L1m45Oz5P4q/70PIvB/el/kx7871OTdXv/kGR93bfTv+77X3olWf/c3t5kvanzAHI4EwCCIwSA4AgBIDhCAAiOEACCIwSA4AgBILhP1jyBzPX+uXH+3P3pc/UPM9f7PzrnR8n6/A0XJuu5cf5HTnooWe+y9Dh6zgf7upL1SzZ/PVk/bvJbyfqegUnJ+px/8mRdGkxW923YlHl9TJwJAMERAkBwhAAQHCEABEcIAMERAkBwhAAQXKPmCeSu58+ty59b9z83j6Csmev2JusLjkvfuX3bPUcm66+9dcSYe2pV9r4Bq36c7n/apvQ8jJee2JGsf3fLz5P17z2xO1nH+HAmAARHCADBEQJAcIQAEBwhAARHCADBEQJAcI2aJ1BWbh5Abj2AnLL3HXhr4WnJ+mErLVmf/kJ6HsKkWwfG3FOrL6+5LFnPzQMoO0/jL27I3PfhvHLHPyfX/ydV9kzAzO4ys34zW9+ybbqZPW5mm4uv06ptE0BVRvN24CeSFnxs23WSVrj7XEkriu8BHISyIeDuKyXt+tjmiyQtLR4vlXRxm/sC0CHj/WCw290PTATfKam7Tf0A6LDSowPu7pJGXAHSzBabWa+Z9Q4o/cEWgM4bbwj0mdksSSq+9o/0RHdf4u497t7TpYnj3B2Aqow3BJZJWlQ8XiTp4fa0A6DTsvMEzOxeSedKmmlm2yTdIOlmSfeZ2RWSXpGUHmAepdx6AWWVHefPKTsP4dA9mXX1M+WX3jsqWZ//evq+BrteS69XMPOJZ9INZOSOT9V/PjlVrzfR1HkI2RBw94UjlOa3uRcANWDaMBAcIQAERwgAwRECQHCEABAcIQAE94laT6Csqsexqx4H33VBeiLB9j87Jlmfc/a2UvvPjbPXPQ+g7J9vU8f5y+JMAAiOEACCIwSA4AgBIDhCAAiOEACCIwSA4JgnMAZNvx5+3+63k/XP3PB0st7/8OeT9b/evClZv+7us5J1fWlPsjx12dT06zPrKeTU/efTVJwJAMERAkBwhAAQHCEABEcIAMERAkBwhAAQHPMEWuTGkcveVyCn7uvVP/yoK1m/cPL76fo3f5SsX9s3L1l/508nJetP3X9qsn50b7W3uav6vgS536+q7svBmQAQHCEABEcIAMERAkBwhAAQHCEABEcIAMExT2AMqr4ePTcOXfU8gt+4PT1OPfelbybrm//w9mT9pqPT/XfZIcn6I4v/K1n/809dkax3/yI9j6DqeSBNlT0TMLO7zKzfzNa3bLvRzLab2drivwuqbRNAVUbzduAnkhYMs/2H7j6v+G95e9sC0CnZEHD3lZJ2daAXADUo88HgVWb2XPF2YVrbOgLQUeMNgdslnShpnqQdkm4Z6YlmttjMes2sd0DVXuABYOzGFQLu3ufu+9x9v6Q7JJ2eeO4Sd+9x954uTRxvnwAqMq4QMLNZLd9eImn9SM8F0GzZeQJmdq+kcyXNNLNtkm6QdK6ZzdPQSvBbJV1ZYY+jlhtHr/p68KqV7b/s9erTZ38lWZ/3t3+SrL99SnqexQu/k55nMP/T+5L19z4zmKzrF+lybh5I2d+vuteLGEk2BNx94TCb76ygFwA1YNowEBwhAARHCADBEQJAcIQAEBwhAAQXaj2Bg30eQdXXu+++PD0PIGfG8+lp4TOeT7/+w/PT4/x/83q6v2+cll5v4NE3epL1Y55MzxPIHZ+q15uoCmcCQHCEABAcIQAERwgAwRECQHCEABAcIQAEF2qeQE7V13uXHYevWtnr6TffdmayvuXSf0jWH3v/8GQ959GXTkrWZ/9nteP42XkcmT//3HoOVeFMAAiOEACCIwSA4AgBIDhCAAiOEACCIwSA4Jgn0EG5ceC67yuw49tnJevd3z02Wb90+s+T9Wv75iXrOXv3p39dB/o+nax3/cfqZL3q9STqmgeQw5kAEBwhAARHCADBEQJAcIQAEBwhAARHCADBMU+gQcquZzDx4tOT9beXfzZZ/9qsp0vtv6z9bsn6vz6Vvm/A3GvS8xSqngeQW48ht55EY9cTMLNjzexJM9tgZs+b2dXF9ulm9riZbS6+Tqu+XQDtNpq3A4OSvuPuJ0s6U9K3zOxkSddJWuHucyWtKL4HcJDJhoC773D3Z4vH70jaKGm2pIskLS2etlTSxVU1CaA6Y/pg0MyOl3SKpNWSut19R1HaKam7rZ0B6IhRh4CZTZH0gKRr3H1Pa83dXZKP8LrFZtZrZr0DSt+wEkDnjSoEzKxLQwFwj7s/WGzuM7NZRX2WpP7hXuvuS9y9x917ujSxHT0DaKPRjA6YpDslbXT3W1tKyyQtKh4vkvRw+9sDULXRzBM4W9LlktaZ2dpi2/WSbpZ0n5ldIekVSZdV02Ichxx1VLL+yyXp+tFTdybrK09elqzP33Bhst4z49Vk/f7nTk3WtacrWT7+4cF0ff9Asp6bB5C9L0BJVf/8qmRDwN1XSRppFsf89rYDoNOYNgwERwgAwRECQHCEABAcIQAERwgAwbGeQBu9fHP6evEjvvhmsn7e7E3J+vLux8fcU6sB35esf/bw15P1+586I1mftj79b8qRL6avt69bbj2AsvMAuO8AgEYiBIDgCAEgOEIACI4QAIIjBIDgCAEgOOYJtNh5zVnJ+rs9HyTrv/+FeseBtw2+m6xf9fLvJeuv3jsnWT9hU73j/Llx+tw4f+lx+sx9Aw5WnAkAwRECQHCEABAcIQAERwgAwRECQHCEABBcqHkCe3/3tGR95te3pesl9/9vL38hWf/8UX3J+pxVV6Zff9PLyfqb56fnAaTvCpAfp8+Nw+fuC5DT1OvxD3acCQDBEQJAcIQAEBwhAARHCADBEQJAcIQAEJy5e/oJZsdKultStySXtMTdbzOzGyX9saQDi9Vf7+7LUz/rcJvuZ9jBezfzsuPcVZvwxJpkfXfF18PnrufPyfWP8VvtK7THd9lwtdFMFhqU9B13f9bMpkpaY2YH7oLxQ3f/QbsaBdB52RBw9x2SdhSP3zGzjZJmV90YgM4Y02cCZna8pFMkrS42XWVmz5nZXWY2rc29AeiAUYeAmU2R9ICka9x9j6TbJZ0oaZ6GzhRuGeF1i82s18x6B7S3DS0DaKdRhYCZdWkoAO5x9wclyd373H2fu++XdIek04d7rbsvcfced+/p0sR29Q2gTbIhYGYm6U5JG9391pbts1qedomk9e1vD0DVRjM6cLakyyWtM7O1xbbrJS00s3kaGjbcKil9nSuARhrN6MAqScONLybnBByMDvZ5AHUre18A1IMZg0BwhAAQHCEABEcIAMERAkBwhAAQHCEABBfqvgM5uXHuqlW9rj7r9mM4nAkAwRECQHCEABAcIQAERwgAwRECQHCEABBc9r4Dbd2Z2euSXmnZNFPSGx1rYOzor5wm99fk3qT29/cZdz9quEJHQ+DXdm7W6+49tTWQQX/lNLm/JvcmdbY/3g4AwRECQHB1h8CSmvefQ3/lNLm/JvcmdbC/Wj8TAFC/us8EANSMEACCIwSA4AgBIDhCAAjufwFbauLb+R51bgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(x_train_adv[0, :].reshape((28, 28)))\n",
    "plt.clim(0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adversarial Training Predicted Label: 7\n"
     ]
    }
   ],
   "source": [
    "prediction = model.predict(x_train_adv[0:1, :])[0]\n",
    "print(\"Adversarial Training Predicted Label: %i\" % prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Benign Test Score: 0.7000\n"
     ]
    }
   ],
   "source": [
    "score = model.score(x_test, y_test)\n",
    "print(\"Benign Test Score: %.4f\" % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAECCAYAAAD+eGJTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAOGElEQVR4nO3df6xf9V3H8ddr7e1lvS2uHaPWUqhjbJHNUcwdbAFNF2SyLaSQbbgmNjWZK1FIwCwqIVlook4k/BCdkhSp6xZgwxWEbHWuaaZIxI7SFFpaFMSirZdeoNOWAf359o97ild27+d7e7/f7znf2/fzkTTf7/e8z/ecd09vX/dzzvdzz3VECEBe72i6AQDNIgSA5AgBIDlCAEiOEACSIwSA5BoJAduX2f4X28/bvqGJHkps77K9zfZW25t7oJ81todtbx+1bK7tDbafqx7n9Fh/q2zvqY7hVtufarC/hbZ/YHuH7WdsX1ct74ljWOivlmPouucJ2J4m6V8lXSppt6QnJC2LiB21NlJge5ekwYh4peleJMn2L0l6TdLXI+JD1bJbJO2LiJurIJ0TEb/XQ/2tkvRaRNzaRE+j2Z4vaX5EbLE9W9KTkq6Q9OvqgWNY6O8q1XAMmxgJXCDp+Yh4ISIOSfqmpKUN9DFlRMSjkva9bfFSSWur52s18kXTiHH66xkRMRQRW6rnByTtlLRAPXIMC/3VookQWCDpP0e93q0a/8ITFJK+b/tJ2yubbmYc8yJiqHr+kqR5TTYzjmttP12dLjR2ujKa7UWSzpe0ST14DN/Wn1TDMeTC4NgujohfkPRJSddUw92eFSPndL02//suSWdLWixpSNJtzbYj2Z4laZ2k6yNi/+haLxzDMfqr5Rg2EQJ7JC0c9fqMalnPiIg91eOwpIc0cgrTa/ZW55LHzymHG+7n/4mIvRFxNCKOSbpbDR9D230a+Q92b0Q8WC3umWM4Vn91HcMmQuAJSefY/lnbMyR9XtIjDfQxJtsD1cUZ2R6Q9AlJ28vvasQjklZUz1dIerjBXn7C8f9clSvV4DG0bUn3SNoZEbePKvXEMRyvv7qOYe2fDkhS9VHHn0iaJmlNRPxh7U2Mw/Z7NfLdX5KmS7qv6f5s3y9piaTTJO2VdJOkv5H0gKQzJb0o6aqIaOTi3Dj9LdHIMDYk7ZJ09ajz77r7u1jSP0raJulYtfhGjZx3N34MC/0tUw3HsJEQANA7uDAIJEcIAMkRAkByhACQHCEAJNdoCPTwlFxJ9NeuXu6vl3uT6u2v6ZFAT/9DiP7a1cv99XJvUo39NR0CABrW1mQh25dJulMjM//+MiJuLq0/w/1xigbeen1YB9Wn/knvv9vorz293F8v9yZ1vr839WMdioMeqzbpEJjMzUFO9dy40JdMan8AJm9TbNT+2DdmCLRzOsDNQYCTQDshMBVuDgKghend3kH1UcdKSTpFM7u9OwAnqJ2RwIRuDhIRqyNiMCIGe/lCDJBVOyHQ0zcHATAxkz4diIgjtq+V9Hf6v5uDPNOxzgDUoq1rAhGxXtL6DvUCoAHMGASSIwSA5AgBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSIwSA5AgBILnp7bzZ9i5JByQdlXQkIgY70RSA+rQVApWPR8QrHdgOgAZwOgAk124IhKTv237S9spONASgXu2eDlwcEXtsny5pg+1nI+LR0StU4bBSkk7RzDZ3B6DT2hoJRMSe6nFY0kOSLhhjndURMRgRg33qb2d3ALpg0iFge8D27OPPJX1C0vZONQagHu2cDsyT9JDt49u5LyK+15GuANRm0iEQES9IOq+DvQBoAB8RAskRAkByhACQHCEAJEcIAMkRAkBynfgpwjRe/eLHivUzlz9frD87PK9YP3Swr1hfcH+5PnP3a8X6sa07inXkxEgASI4QAJIjBIDkCAEgOUIASI4QAJIjBIDkmCdwAn73d+4r1j8z8KPyBs5us4El5fKuI68X63e+/PE2G5jafjh8VrE+cNtPFevTNz7ZyXZ6BiMBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSc0TUtrNTPTcu9CW17a/TfvzZC4v1Vz5cztQ5O8vH+kc/52J9xof/u1i/5UMPFuuXvvONYv27r88q1j89s3y/gna9EYeK9U0HB4r1Jaccbmv/7/vu1cX6+1c+0db2m7QpNmp/7BvzC4yRAJAcIQAkRwgAyRECQHKEAJAcIQAkRwgAyXE/gRMw8O1NLertbf/U9t6uP/vpJcX6H1y0qLz/fyj/3oRblrzvBDs6MdPfOFasDzw9VKy/+9F1xfrPz2jxext2lesnq5YjAdtrbA/b3j5q2VzbG2w/Vz3O6W6bALplIqcDX5N02duW3SBpY0ScI2lj9RrAFNQyBCLiUUn73rZ4qaS11fO1kq7ocF8AajLZC4PzIuL4CdpLksq/ZA9Az2r704EY+QmkcX8yxvZK25ttbz6sg+3uDkCHTTYE9tqeL0nV4/B4K0bE6ogYjIjBPvVPcncAumWyIfCIpBXV8xWSHu5MOwDq1nKegO37NXLH+9Ns75Z0k6SbJT1g+wuSXpR0VTebxMQceWlvsT6wrlw/2mL7A99+9QQ76qy9v/GxYv2DM8pfzrfu+0CxvuivXijWjxSrU1fLEIiIZeOUpu7dQQC8hWnDQHKEAJAcIQAkRwgAyRECQHKEAJAc9xNAz5h+1sJi/as3frVY7/O0Yv2v7/zlYv3dQ48X6ycrRgJAcoQAkBwhACRHCADJEQJAcoQAkBwhACTHPAH0jGd/e0Gx/pF+F+vPHHqjWJ+74/UT7ikDRgJAcoQAkBwhACRHCADJEQJAcoQAkBwhACTHPAHU5uCnP1Ksb/nsHS22UP4NVr953XXF+jv/6Ycttp8TIwEgOUIASI4QAJIjBIDkCAEgOUIASI4QAJJjngBq8x+fLH/PmeXyPIBl/35psT7ze08V61Gs5tVyJGB7je1h29tHLVtle4/trdWfT3W3TQDdMpHTga9JumyM5XdExOLqz/rOtgWgLi1DICIelbSvhl4ANKCdC4PX2n66Ol2Y07GOANRqsiFwl6SzJS2WNCTptvFWtL3S9mbbmw/r4CR3B6BbJhUCEbE3Io5GxDFJd0u6oLDu6ogYjIjBvhY/BQagfpMKAdvzR728UtL28dYF0NtazhOwfb+kJZJOs71b0k2SltherJGPXndJurqLPWKKeMfs2cX68l98rFjff+zNYn34K+8t1vsPPlGsY2wtQyAilo2x+J4u9AKgAUwbBpIjBIDkCAEgOUIASI4QAJIjBIDkuJ8AOua5VR8s1r9z2l8U60uf+0yx3r+eeQDdwEgASI4QAJIjBIDkCAEgOUIASI4QAJIjBIDkmCeACfufX/tosf70r/5psf5vRw4X66/98RnFer+GinVMDiMBIDlCAEiOEACSIwSA5AgBIDlCAEiOEACSY54A3jJ9wc8U69d/+VvFer/LX06ff2p5sf6ev+V+AU1gJAAkRwgAyRECQHKEAJAcIQAkRwgAyRECQHLME0jE08v/3Od9Z3ex/rlZrxbr9x44vVif9+Xy95xjxSq6peVIwPZC2z+wvcP2M7avq5bPtb3B9nPV45zutwug0yZyOnBE0pci4lxJH5V0je1zJd0gaWNEnCNpY/UawBTTMgQiYigitlTPD0jaKWmBpKWS1larrZV0RbeaBNA9J3Rh0PYiSedL2iRpXkQcv+nbS5LmdbQzALWYcAjYniVpnaTrI2L/6FpEhKQY530rbW+2vfmwDrbVLIDOm1AI2O7TSADcGxEPVov32p5f1edLGh7rvRGxOiIGI2KwT/2d6BlAB03k0wFLukfSzoi4fVTpEUkrqucrJD3c+fYAdNtE5glcJGm5pG22t1bLbpR0s6QHbH9B0ouSrupOi+iY8z5QLP/+6d9oa/N//pXPFevveurxtraP7mgZAhHxmCSPU76ks+0AqBvThoHkCAEgOUIASI4QAJIjBIDkCAEgOe4ncBKZdu77i/WV32xvPte5a64p1hd945/b2j6awUgASI4QAJIjBIDkCAEgOUIASI4QAJIjBIDkmCdwEnn2t8p3fb985v5ivZUz/v5QeYUY8w5z6HGMBIDkCAEgOUIASI4QAJIjBIDkCAEgOUIASI55AlPIm5dfUKxvvPy2FluY2blmcNJgJAAkRwgAyRECQHKEAJAcIQAkRwgAyRECQHIt5wnYXijp65LmSQpJqyPiTturJH1R0svVqjdGxPpuNQrpvy6aVqyfOb29eQD3Hji9WO/bX76fAHcTmJomMlnoiKQvRcQW27MlPWl7Q1W7IyJu7V57ALqtZQhExJCkoer5Ads7JS3odmMA6nFC1wRsL5J0vqRN1aJrbT9te43t8r2tAPSkCYeA7VmS1km6PiL2S7pL0tmSFmtkpDDmxHXbK21vtr35sA52oGUAnTShELDdp5EAuDciHpSkiNgbEUcj4pikuyWN+dMtEbE6IgYjYrBP/Z3qG0CHtAwB25Z0j6SdEXH7qOXzR612paTtnW8PQLdN5NOBiyQtl7TN9tZq2Y2SltlerJFPhnZJurorHQLoqol8OvCYJI9RYk7AFPNHr55brD/+K4uK9Rja1sFu0CuYMQgkRwgAyRECQHKEAJAcIQAkRwgAyRECQHKOGn+n/KmeGxf6ktr2B2DEptio/bFvrPk+jASA7AgBIDlCAEiOEACSIwSA5AgBIDlCAEiu1nkCtl+W9OKoRadJeqW2Bk4c/bWnl/vr5d6kzvd3VkS8Z6xCrSHwEzu3N0fEYGMNtEB/7enl/nq5N6ne/jgdAJIjBIDkmg6B1Q3vvxX6a08v99fLvUk19tfoNQEAzWt6JACgYYQAkBwhACRHCADJEQJAcv8LId/VeNhqNOUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(x_test[0, :].reshape((28, 28)))\n",
    "plt.clim(0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Benign Test Predicted Label: 7\n"
     ]
    }
   ],
   "source": [
    "prediction = model.predict(x_test[0:1, :])[0]\n",
    "print(\"Benign Test Predicted Label: %i\" % prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adversarial Test Score: 0.1000\n"
     ]
    }
   ],
   "source": [
    "score = model.score(x_test_adv, y_test)\n",
    "print(\"Adversarial Test Score: %.4f\" % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAECCAYAAAD+eGJTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAP1UlEQVR4nO3df6yW9XnH8c+FHKACCgQ5RUoLUlyxy4ruRLthGi3RMJNFbTZX/nA0a4p/1LRmzVZDssg/S9xWdc2yueBgpdXamaBiN7PJWBdr2jEPlghInegghRzOwZ52YHXIj2t/nFt3qud878P5PvePc673KyHnOfd1nue+vA98vO/n+72/j7m7AMQ1pekGADSLEACCIwSA4AgBIDhCAAiOEACCayQEzGyNmb1sZgfN7O4mekgxs0NmttfM9phZbwv62WJmA2a2b9i2eWa2w8xeKb7ObVl/G83saHEM95jZTQ32t9jMvmdmL5nZfjP7crG9Fccw0V8tx9DqnidgZhdI+i9JN0g6Iul5SWvd/aVaG0kws0OSetz99aZ7kSQz+5SkNyR9091/tdj255IG3f3eIkjnuvtXW9TfRklvuPvXmuhpODNbKGmhu79gZrMl7ZZ0i6TPqQXHMNHfbarhGDZxJnC1pIPu/pq7vy3pO5JubqCPCcPdn5U0+J7NN0vaWjzeqqG/NI0Ypb/WcPc+d3+heHxS0gFJi9SSY5jorxZNhMAiST8Z9v0R1fgfPEYu6Rkz221m65tuZhTd7t5XPD4mqbvJZkZxp5m9WFwuNHa5MpyZLZF0paRdauExfE9/Ug3HkDcGR3atu18l6bckfbE43W0tH7qma9v87wclLZO0UlKfpPuabUcys1mStkm6y91PDK+14RiO0F8tx7CJEDgqafGw7z9UbGsNdz9afB2Q9ISGLmHapr+4lnznmnKg4X5+ibv3u/tZdz8n6SE1fAzNrEtD/8AecffHi82tOYYj9VfXMWwiBJ6XtNzMlprZNEmflfRUA32MyMxmFm/OyMxmSrpR0r70sxrxlKR1xeN1krY32Mv7vPOPq3CrGjyGZmaSNks64O73Dyu14hiO1l9dx7D20QFJKoY6/lLSBZK2uPuf1t7EKMzsMg3931+Spkr6dtP9mdmjkq6TNF9Sv6R7JD0p6TFJH5Z0WNJt7t7Im3Oj9Hedhk5jXdIhSXcMu/6uu79rJX1f0l5J54rNGzR03d34MUz0t1Y1HMNGQgBAe/DGIBAcIQAERwgAwRECQHCEABBcoyHQ4im5kugvV5v7a3NvUr39NX0m0OpfhOgvV5v7a3NvUo39NR0CABqWNVnIzNZI+rqGZv79nbvfm/r5aTbdZ2jmu9+f1il1afq49181+svT5v7a3JvU+f7+V7/Q237KRqqNOwTGszjIRTbPr7HV49ofgPHb5Tt1wgdHDIGcywEWBwEmgZwQmAiLgwAoMbXqHRRDHeslaYYurHp3AM5TzpnAmBYHcfdN7t7j7j1tfiMGiConBFq9OAiAsRn35YC7nzGzOyX9i/5/cZD9HesMQC2y3hNw96clPd2hXgA0gBmDQHCEABAcIQAERwgAwRECQHCEABBc5dOGMXn8/PbfaLqFpFlH307Wp/7b7po6mVg4EwCCIwSA4AgBIDhCAAiOEACCIwSA4AgBIDjmCWDSeGPRtGR9Tk19TDScCQDBEQJAcIQAEBwhAARHCADBEQJAcIQAEBzzBPCuM5/+9aZbqFTueghl6xWUaet6BpwJAMERAkBwhAAQHCEABEcIAMERAkBwhAAQHPMEhml6Xf053/ph1vNzx/nL7sdvWtk4fVn/k3WcP1dWCJjZIUknJZ2VdMbdezrRFID6dOJM4Hp3f70DrwOgAbwnAASXGwIu6Rkz221m6zvREIB65V4OXOvuR81sgaQdZvZjd392+A8U4bBekmbowszdAei0rDMBdz9afB2Q9ISkq0f4mU3u3uPuPV2anrM7ABUYdwiY2Uwzm/3OY0k3StrXqcYA1CPncqBb0hNm9s7rfNvd/7kjXVWkbB7ARP98+7L+cucR5I7TN/36GNm4Q8DdX5P0iQ72AqABDBECwRECQHCEABAcIQAERwgAwRECQHCTaj2BsnHw3Pv1p1yYnvZ8+A9XJusX9nmyfmqeJeuX/sUPkvWqNX6/fsPrPUxWnAkAwRECQHCEABAcIQAERwgAwRECQHCEABDcpJonUPX9/i//zYpkfdnin1S6/3NrFifr3R84maw/vGRz1v4v2/EHyfpgX+bKUTfmzQM4u/BUsv5Pn/6rZP33PvelrP1PVJwJAMERAkBwhAAQHCEABEcIAMERAkBwhAAQnLmn73HvpItsnl9jqyt7/dx19XPnGdiVH0/W/Uf7k/W3bnnfBzj9kuMr09M6Ti9/K1k/eP3fJ+uf2ntrsn7tgleT9VzPDSxL1vt+enGy/rsrXsja/6PPX5OsL/h++vi3+XMrdvlOnfDBERes4EwACI4QAIIjBIDgCAEgOEIACI4QAIIjBIDgQq0nkDuPoOz5uePAH3jyP5P1Dz+Z9fK66ZIb0vs//t/J+p7uDybrZ/sHkvULuhek99+f3v9lyar042e7k/WPze5P1pdsS8+ZmXI673MV2qr0TMDMtpjZgJntG7ZtnpntMLNXiq9zq20TQFXGcjnwDUlr3rPtbkk73X25pJ3F9wAmoNIQcPdnJQ2+Z/PNkrYWj7dKuqXDfQGoyXjfGOx2977i8TFJ6YsxAK2VPTrgQ3cgjfqOipmtN7NeM+s9rfRCkADqN94Q6DezhZJUfB31bWF33+TuPe7e06XM1WgBdNx4Q+ApSeuKx+skbe9MOwDqVjpPwMwelXSdpPlmdkTSPZLulfSYmX1e0mFJt1XZ5Fj9PPfz60ueX3a/eNudPX487/kl8wDK5lGcKXn9qWXzDD7+K8n68tkvJ+tPHPxEsj7tY9OS9RmDmWtvlPz9mvOtH+a9/jiVhoC7rx2lVN3qIABqw7RhIDhCAAiOEACCIwSA4AgBIDhCAAhuUq0nkKupcdpOyV3vIHe9hTcWpcfZy8wpqU/56xPJ+p7BDyXrSzem53n89Kr6PoOjTTgTAIIjBIDgCAEgOEIACI4QAIIjBIDgCAEgOOYJtEj2eghlMtdLyJ4HUDIP42fr0v1dfPZIsn7sxOxk/dTtFyfrc/cny5MWZwJAcIQAEBwhAARHCADBEQJAcIQAEBwhAATHPIFhysbpy8a5c+/Hzx2nz31+7jyAMlNmzkzWp63tT9Z3rPhusr70u19I1hdkzgMoO75l6zWUyV0PYrw4EwCCIwSA4AgBIDhCAAiOEACCIwSA4AgBIDjmCZyHqu/3LxuHrvp+/1xl/R3/7K8l63O70usFLN2+Pllfsj39uQFT3q52nH+iKj0TMLMtZjZgZvuGbdtoZkfNbE/x56Zq2wRQlbFcDnxD0poRtj/g7iuLP093ti0AdSkNAXd/VtJgDb0AaEDOG4N3mtmLxeXC3I51BKBW4w2BByUtk7RSUp+k+0b7QTNbb2a9ZtZ7WqfGuTsAVRlXCLh7v7ufdfdzkh6SdHXiZze5e4+793Rp+nj7BFCRcYWAmS0c9u2tkvaN9rMA2s3c02OrZvaopOskzZfUL+me4vuVklzSIUl3uHtf2c4usnl+ja3OajhH5ev6V6zqeQK598u/vj59fHdvfDBZX/XiZ5L1iz5zLFk/9+abyXrbf/9l61Xk2OU7dcIHbaRa6WQhd187wubN2V0BaAWmDQPBEQJAcIQAEBwhAARHCADBEQJAcKHWEygbh80dR84d5y1bd77p9QLKdL2ZnnNSZu6Mt5L1tz65Iuv1y5TNkyjT9t/PaDgTAIIjBIDgCAEgOEIACI4QAIIjBIDgCAEguFDzBMpUeT/3WLR9nHnq0o8k6zN+P32/f5nX/nVpsr6w4uXpcj93YE6H+qgbZwJAcIQAEBwhAARHCADBEQJAcIQAEBwhAATHPIEala0XkCv3fvi3LulK1nc+vD1ZP+vnkvXN/3Npsj77cN56BLnzLMp+P7nzCMrWq2hqngpnAkBwhAAQHCEABEcIAMERAkBwhAAQHCEABMc8gRbJHecv41NH/Hj6d/3ggb/Nev1HTi5I1h9b8cH0C9yeLpfNAyg7fmXPz50HUKbp9SpGU3omYGaLzex7ZvaSme03sy8X2+eZ2Q4ze6X4Orf6dgF02lguB85I+oq7XyHpk5K+aGZXSLpb0k53Xy5pZ/E9gAmmNATcvc/dXygen5R0QNIiSTdL2lr82FZJt1TVJIDqnNcbg2a2RNKVknZJ6nb3vqJ0TFJ3RzsDUIsxh4CZzZK0TdJd7n5ieM3dXdKId3+Y2Xoz6zWz3tMVLxQJ4PyNKQTMrEtDAfCIuz9ebO43s4VFfaGkgZGe6+6b3L3H3Xu6NL0TPQPooLGMDpikzZIOuPv9w0pPSVpXPF4nKX2fKYBWGss8gVUaGsHda2Z7im0bJN0r6TEz+7ykw5Juq6bF9shdD6BsHLrq9QZ+9tH0OHnZegB/MrAyWf+Hf//NZP2j+o9kPXeeRNnxnaifC1C10hBw9+ckjTbLZHVn2wFQN6YNA8ERAkBwhAAQHCEABEcIAMERAkBwrCdwHnLH+ate1/6CKy5P1n/08OZkfUN/eh7Atn9clazPfyVZzla6nkDFx7fs91fW34RdTwDA5EYIAMERAkBwhAAQHCEABEcIAMERAkBwzBOYQMo+3/53vvpMpfvv+kX6cwtmHclbPq5snL1pbe9vvDgTAIIjBIDgCAEgOEIACI4QAIIjBIDgCAEgOOYJdFDV96sPrDqbrP/RvFez9l9m0Z+l74c/c/1V6Xrm5yrkfi5B1es5TFScCQDBEQJAcIQAEBwhAARHCADBEQJAcIQAEFzpPAEzWyzpm5K6JbmkTe7+dTPbKOkLko4XP7rB3Z+uqtE2qHqcuez5l/7xZVmvf8OB307WDx7qTtYv995kPfe/v+xzA5rW1s8NyDWWyUJnJH3F3V8ws9mSdpvZjqL2gLt/rbr2AFStNATcvU9SX/H4pJkdkLSo6sYA1OO83hMwsyWSrpS0q9h0p5m9aGZbzGxuh3sDUIMxh4CZzZK0TdJd7n5C0oOSlklaqaEzhftGed56M+s1s97TyluDDkDnjSkEzKxLQwHwiLs/Lknu3u/uZ939nKSHJF090nPdfZO797h7T5emd6pvAB1SGgJmZpI2Szrg7vcP275w2I/dKmlf59sDULWxjA6sknS7pL1mtqfYtkHSWjNbqaFhw0OS7qikQwCVGsvowHOSRlpwftLNCci9373tXt1/abJ++Zd2JettF3U9gFzMGASCIwSA4AgBIDhCAAiOEACCIwSA4AgBIDhz99p2dpHN82tsdW37AzBkl+/UCR8cab4PZwJAdIQAEBwhAARHCADBEQJAcIQAEBwhAARX6zwBMzsu6fCwTfMlvV5bA+eP/vK0ub829yZ1vr+PuPslIxVqDYH37dys1917GmugBP3laXN/be5Nqrc/LgeA4AgBILimQ2BTw/svQ3952txfm3uTauyv0fcEADSv6TMBAA0jBIDgCAEgOEIACI4QAIL7P7BBs/RFVjJ6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(x_test_adv[0, :].reshape((28, 28)))\n",
    "plt.clim(0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adversarial Test Predicted Label: 9\n"
     ]
    }
   ],
   "source": [
    "prediction = model.predict(x_test_adv[0:1, :])[0]\n",
    "print(\"Adversarial Test Predicted Label: %i\" % prediction)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Investigate dependence on attack budget eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO3deZgU9bn28e8zPRsw7IsLOxEVEBUY2XOymcQVFMQDBCMuQVzw5CRvEnM0bzyaKyYmJgZEEMVdA4oRcY/GJcomg6IIirIYAYmMsoMwDDznjy60HWfpgampnqn7c119TXdVdfdtyczd1VX1K3N3REQkvrKiDiAiItFSEYiIxJyKQEQk5lQEIiIxpyIQEYk5FYGISMyFVgRmdqeZbTSztyuYb2Y20cxWmtlbZtY7rCwiIlKxMLcI7gZOqWT+qUDX4DYOmBJiFhERqUBoReDu/wQ2VbLIUOBeT1oANDOzI8LKIyIi5cuO8L3bAmtTHq8Lpm0ou6CZjSO51UCjRo36HHvssbUSUESkvli8ePEn7t66vHlRFkHa3H0aMA2gsLDQi4qKIk4kIlK3mNm/KpoX5VFD64H2KY/bBdNERKQWRVkEc4AfBkcP9Qe2uvtXvhYSEZFwhfbVkJn9Ffgm0MrM1gG/BnIA3H0q8BRwGrAS2AVcEFYWERGpWGhF4O6jqpjvwOVhvb+IiKRHZxaLiMScikBEJOZUBCIiMaciEBGJORWBiEjMqQhERGJORSAiEnMqAhGRmFMRiIjEnIpARCTmVAQiIjGnIhARiTkVgYhIzKkIRERiTkUgIhJzKgIRkZhTEYiIxJyKQEQk5lQEIiIxpyIQEYk5FYGISMypCEREYk5FICIScyoCEZGYUxGIiMScikBEJOZUBCIiMaciEBGJORWBiEjMqQhERGJORSAiEnMqAhGRmFMRiIjEnIpARCTmVAQiIjEXahGY2SlmtsLMVprZVeXM72BmL5rZG2b2lpmdFmYeERH5qtCKwMwSwGTgVKA7MMrMupdZ7BrgIXfvBYwEbg0rj4iIlC/MLYK+wEp3X+3uJcAMYGiZZRxoEtxvCnwUYh4RESlHmEXQFlib8nhdMC3VtcAYM1sHPAVMKO+FzGycmRWZWVFxcXEYWUVEYivqncWjgLvdvR1wGnCfmX0lk7tPc/dCdy9s3bp1rYcUEanPwiyC9UD7lMftgmmpLgIeAnD3+UA+0CrETCIiUkaYRbAI6Gpmnc0sl+TO4DlllvkQ+A6AmXUjWQT67kdEpBaFVgTuXgpcATwLvEPy6KBlZnadmQ0JFvsp8CMzexP4KzDW3T2sTCIi8lXZYb64uz9Fcidw6rT/n3J/OTAozAwiIlK5qHcWi4hIxFQEIiIxpyIQEYk5FYGISMypCEREYk5FICIScyoCEZGYUxGIiMScikBEJOZUBCIiMaciEBGJORWBiEjMqQhERGJORSAiEnMqAhGRmFMRiIjEnIpARCTmVAQiIjEXqyIo3bc/6ggiIhknNkXwzNsbGHLLXD74ZGfUUUREMkpsiiA/J8H6LZ9x5i2v8vdl/446johIxohNEXzzmDY8MWEwnVo2Ytx9i/nd0+/qqyIREWJUBADtWzTk4fEDGN2vA1NfXsWY6Qsp3r4n6lgiIpGKVRFA8iui357dk5tGnMCStVs4feIrLPpgU9SxREQiE7siOGB4n3bMvnwQjfKyGTltAXe8shp3jzqWiEiti20RABx7eBMeu2IQJ3drw2+efIfLHnid7bv3Rh1LRKRWVVkEZpaojSBRaZKfw9Qxfbj6tG78ffnHDL1lLiv+vT3qWCIitSadLYL3zewPZtY99DQRMTN+9B9dePDifmzfU8pZk+fy6Bvroo4lIlIr0imCE4D3gDvMbIGZjTOzJiHnikS/Li158srB9GzXlP+e+SbXzF7KntJ9UccSEQlVlUXg7tvd/XZ3Hwj8Avg1sMHM7jGzo0JPWMvaNM7nwYv7cck3unD/gg85d+p81m3eFXUsEZHQpLWPwMyGmNmjwM3ATUAX4HHgqZDzRSI7kcUvT+3Gbef1YXXxTs6Y9CovrdgYdSwRkVCktY8AGAr8wd17ufuf3P1jd58FPBNuvGh9v8fhPD5hMIc3yeeCuxfx5+feY99+HWIqIvWLVXXsvJkVuPuOWspTpcLCQi8qKqrV9/ysZB/XzH6bR15fx9e7tuIvI3vRolFurWYQETkUZrbY3QvLm5fOFsFkM2uW8mLNzezOGktXBzTITfDHEcdzw7CeLFyziTMmvsKStVuijiUiUiPSKYLj3f3zv3ruvhnolc6Lm9kpZrbCzFaa2VUVLHOumS03s2Vm9mB6sWufmTGqbwceGT+QrCxjxNR53Df/A52NLCJ1XjpFkGVmzQ88MLMWQHZVTwpORJsMnAp0B0aVPRfBzLoCvwQGuXsP4MfVyB6Jnu2a8sSEwXy9a2t+9dgyfjxzCbtKSqOOJSJy0NIpgpuA+WZ2vZn9BpgH3JjG8/oCK919tbuXADNI7nRO9SNgcrCVgbvXiUNzmjXM5Y4fFvKz7x/D429+xFmT57KqOGN2o4iIVEs65xHcCwwHPgb+DQxz9/vSeO22wNqUx+uCaamOBo42s7nByWqnlPdCwUlsRWZWVFxcnMZbhy8ry7j8W0dx74X9+GRHCUMmvcpTSzdEHUtEpNrSGnTO3ZcBDwFzgB1m1qGG3j8b6Ap8ExgF3J66Yzrl/ae5e6G7F7Zu3bqG3rpmDO7aiievHMwxhzfmsgde57rHl7NXF7wRkToknRPKhpjZ+8Aa4GXgA+DpNF57PdA+5XG7YFqqdcAcd9/r7mtIDmXRNY3XzihHNG3AjHEDGDuwE3fOXcOoaQv499bdUccSEUlLOlsE1wP9gffcvTPwHWBBGs9bBHQ1s85mlguMJLlFkWo2ya0BzKwVya+KVqcXPbPkZmdx7ZAeTBrVi+UbtnHGpFeYt/KTqGOJiFQpnSLY6+6fkjx6KMvdXwTKPSkhlbuXAlcAzwLvAA+5+zIzu87MhgSLPQt8ambLgReBnwXvVWedecKRzLliEM0a5jJm+kImv7iS/TobWUQyWDpnFj8PnAXcALQCNgInBYPQ1booziw+GDv3lHLV35by+JsfcXK3Ntw04kSaNsyJOpaIxNShnlk8FNgF/DfJsYVWAWfWXLz6qVFeNhNHnsi1Z3bn5feKOeOWV3h7/daoY4mIfEWlRRCcFPaEu+9391J3v8fdJ9b1r29qi5kxdlBnZl4ygNJ9zrAp85i56MOoY4mIfEmlReDu+4D9Zta0lvLUS707NOeJCYPp26kFv3hkKT+f9Sa79+qCNyKSGaocKgLYASw1s+eAnQcmuvuVoaWqh1oW5HHPhX35y/PvMfGFlby9fhtTxvSmY8tGUUcTkZhLZx/B34BfAf8EFqfcpJoSWcZPvncMd409ifVbPuOMSa/y3PKPo44lIjFX5VFDmaauHDVUlbWbdnHZA6+zdP1WLv3m1/jpd48mO5HWid4iItV2SEcNmdkaM1td9lbzMeOlfYuGPDx+AKP7dWDKS6s4b/prFG/fE3UsEYmhdPYRpDZIPjACaBFOnHjJz0nw27N70qdDc66evZTTJ77C5B/05qROWr0iUnvSGX3005Tbene/GTi9FrLFxvA+7Xj0skE0zE0wctoC7nhltS54IyK1Jp2vhnqn3ArNbDzpbUlINXQ7oglzJgzm5G5t+M2T73D5g6+zfffeqGOJSAyk8wf9ppT7pSRHIT03nDjx1iQ/h6lj+nD7K6v5/TMreHfDXKaM6cMxhzeOOpqI1GM6aihDLVz9KVf89Q127C7lhmE9OatX2Wv6iIik71CPGvpt6sVizKx5cMlKCVG/Li15csJgerZryo9nLuGa2UvZU6qzkUWk5qVz4Pqp7r7lwIPg+sKnhRdJDmjTJJ8HL+7HJf/RhfsXfMi5U+ezbvOuqGOJSD2TThEkzCzvwAMzawDkVbK81KDsRBa/PK0bU8f0YXXxTs6Y9CovrdgYdSwRqUfSKYIHgH+Y2UVmdhHwHHBPuLGkrFOOO5w5EwZzeJN8Lrh7ETc//54ueCMiNSKd8wh+D/wG6Bbcrnf3G8MOJl/VuVUjHr1sEGf3asvNz7/P2LsXsWlnSdSxRKSOS+cKZZ2BDe6+O3jcADjM3T8IP95XxeWoocq4O399bS3XzllG68Z5TP5Bb05s36zqJ4pIbB3qFcoeBvanPN4XTJOImBmj+3XgkUsHYgYjps7jvvkf6GxkETko6RRBtrt//v1DcD83vEiSrp7tmvLEhMEMPqoVv3psGf89cwm7SkqjjiUidUw6RVBsZkMOPDCzocAn4UWS6mjWMJfp55/E//ve0Tz25kecNXkuq4p3RB1LROqQdIpgPPA/Zvahma0FfgGMCzeWVEdWlnHFt7ty74V9+WRHCUNvmctTSzdEHUtE6oh0jhpa5e79ge5AN3cfiIahzkhf79qaJyYMputhBVz2wOtc/8Ry9u7bX/UTRSTWqnNJrA7AL8zsfWBKSHnkEB3ZrAEzxw1g7MBOTH91DaOmLeDfW3dHHUtEMlilRWBmnczsl2b2FnAfcCnw3YoOQZLMkJudxbVDejBxVC+Wb9jGGZNeYd4q7dYRkfJVWARmNh94kuRQ1cPdvQ+wParzB6T6hpxwJHOuGETTBjmMuWMht760Umcji8hXVLZF8DHQGDgMaB1M01+ROuaoNo157IrBnNbzCG58ZgXj7lvM1s90wRsR+UKFReDuZwE9gcXAtWa2BmhuZn1rK5zUjIK8bCaN6sW1Z3bnpRUbOXPSqyz7aGvUsUQkQ1S6j8Ddt7r7Xe7+PaAf8Cvgz8FhpFKHmBljB3Vm5iUDKCndz7Bb5/HQIv1vFJFqHDXk7hvd/RZ3HwQMDjGThKhPx+Y8eeVgTurUgp8/8hY/n/Umu/fqgjcicVadw0c/5+7/qukgUntaFuRxz4V9mfDto3ioaB3Dbp3Hh5/qgjcicXVQRSB1XyLL+On3juHOsYWs3/IZp096heeXfxx1LBGJQDrXLB6UzjSpm7597GE8MWEwHVs25OJ7i7jxmXcp1dnIIrGSzhbBpDSnSR3VvkVDZo0fyKi+Hbj1pVWcN/01irfviTqWiNSS7IpmmNkAYCDQ2sx+kjKrCZAIO5jUrvycBDcM60mfjs25+tGlnDHpFSaP7k1hJw0rJVLfVbZFkAsUkCyLxim3bcA56by4mZ1iZivMbKWZXVXJcsPNzM1MQ1dE7Jw+7Zh9+SAa5CQYOW0Bs99YH3UkEQlZhVsE7v4y8LKZ3X3gKCEzywIK3H1bVS9sZglgMvBdYB2wyMzmuPvyMss1Bv4LWHjw/xlSk7od0YQ5EwZzyb2L+fHMJWzaWcKFgztHHUtEQpLOPoIbzKyJmTUC3gaWm9nP0nheX2Clu68Ormo2AxhaznLXA78HNERmBmmSn8NdF5zEKT0O57onlvOHZ9/VpTBF6ql0iqB7sAVwFvA00Bk4L43ntQVST11dF0z7nJn1Btq7+5OVvZCZjTOzIjMrKi4uTuOtpSbk5ySY/IPejOrbgckvruKXf1uqI4pE6qF0iiDHzHJIFsEcd99LDQw+F3zN9Cfgp1Ut6+7T3L3Q3Qtbt25d1eJSgxJZxm/PPo4J3z6KGYvWcvmDr+tMZJF6Jp0iuA34AGgE/NPMOpLcYVyV9UD7lMftgmkHNAaOA14ysw+A/sAc7TDOPGbJk89+fWZ3nl32MWPveo3tuzWCqUh9kc6lKie6e1t3P82T/gV8K43XXgR0NbPOZpYLjATmpLzuVndv5e6d3L0TsAAY4u5FB/efImG7YFBn/jLyRIo+2MzIaQt0roFIPZHOmcWHmdl0M3s6eNwdOL+q57l7KXAF8CzwDvCQuy8zs+vMbMgh5paIDD2xLXecX8jq4p2cM1VjFInUB1bVkSBBAdwFXO3uJ5hZNvCGu/esjYBlFRYWelGRNhqi9saHm7ng7kXkJLK498K+dDuiSdSRRKQSZra4ossMV3apygPnGLRy94eA/fD5J33tLYy5Xh2aM2v8ALKzjHNvm89razZFHUlEDlJlXw29FvzcaWYtCY4UMrP+gC5vJRzVpjGzLh1Im8Z5nDd9Ic9p9FKROqmyIrDg509I7uT9mpnNBe4FJoQdTOqGts0a8PD4gRx7RBPG37+Yh4p01TORuqbCISb48mBzjwJPkSyHPcDJwFshZ5M6okWjXB68uB/j71/Mz2e9xaadJYz/xteijiUiaapsiyBBctC5xiTPIcgOpjUMpol8rlFeNtPPP4kzjj+C3z39Lr996h3279eQFCJ1QWVbBBvc/bpaSyJ1Xm52FhNH9qJlo1ym/XM1n+4o4XfDe5KT0IXwRDJZZUVglcwTKVdWlnHtkB60LMjjT8+9x5ZdJdwyujcNcnUJC5FMVdlHte/UWgqpV8yMK7/Tld+cdRwvrNjIedMXsnWXhqQQyVQVFoG768BwOSRj+ndk8ujevLVuK+feNp+Pt2mkcZFMpC9vJVSn9TyCuy44iXWbdzHs1nmsLt4RdSQRKUNFIKEbdFQrZowbwO69+xgxdT5L1+l8RJFMoiKQWtGzXVMeHj+A/JwEI6fNZ97KT6KOJCIBFYHUmi6tC/jbZQNp17whY+9axFNLN0QdSURQEUgtO6xJPg9dMoDj2zXl8gdf5/4F/4o6kkjsqQik1jVtmMN9F/XjW8e04ZrZbzPxH+9T1XDoIhIeFYFEokFugtvO68Ow3m3503Pvce2cZRqSQiQilZ1ZLBKqnEQWfzznBFo2yuX2V9awaddebhpxArnZ+nwiUptUBBKprCzj6tO707Igj989/S5bdpUwdUwfGuXpn6ZIbdFHL8kI47/xNW4cfjxzV37C6DsWsmlnSdSRRGJDRSAZ49yT2jN1TB/e2bCNEVPnsX7LZ1FHEokFFYFklO/1OJz7LuzLxm17OGfKPFZu3B51JJF6T0UgGadfl5bMvGQAe/c550ydzxsfbo46kki9piKQjNT9yCb87dKBNG2Qw+jbF/Lye8VRRxKpt1QEkrE6tGzIw+MH0LlVIy66exGPLVkfdSSReklFIBmtTeN8ZlzSnz4dm/NfM5Zw19w1UUcSqXdUBJLxmuTncM+Fffle98P438eXc9PfV2hICpEapCKQOiE/J8GtP+jNyJPaM+mFlfzPo2+zT0NSiNQInb4pdUZ2IosbhvWkZUEuk19cxeadJdw88kTycxJRRxOp07RFIHWKmfGz7x/Lr87ozjPL/s0Fdy1i++69UccSqdNUBFInXTS4Mzf/54ks+mATI6ctoHj7nqgjidRZKgKps87q1Zbbzy9kVfEORkydx9pNu6KOJFInqQikTvvWMW144OL+bN61l2FT5vHOhm1RRxKpc1QEUuf16dich8cPIGHGubfN57U1m6KOJFKnqAikXjj6sMbMunQArQvyOG/6Qp5f/nHUkUTqjFCLwMxOMbMVZrbSzK4qZ/5PzGy5mb1lZv8ws45h5pH6rV3z5JAUxxzemEvuX8zDRWujjiRSJ4RWBGaWACYDpwLdgVFm1r3MYm8Ahe5+PDALuDGsPBIPLQvyePBH/RnQpSU/m/UWt728KupIIhkvzC2CvsBKd1/t7iXADGBo6gLu/qK7HzjUYwHQLsQ8EhMFedlMH1vI6ccfwQ1Pv8tvn3pHQ1KIVCLMM4vbAqnb5uuAfpUsfxHwdHkzzGwcMA6gQ4cONZVP6rG87AQTR/aiRcNcpv1zNZ/uKOH3w3uSndBuMZGyMmKICTMbAxQC3yhvvrtPA6YBFBYW6qOdpCWRZVw3tActC3K5+fn32bKrhFtG96ZBroakEEkV5sej9UD7lMftgmlfYmYnA1cDQ9xdp4dKjTIzfnzy0Vw/tAcvrNjID+9cyNZdGpJCJFWYRbAI6Gpmnc0sFxgJzEldwMx6AbeRLIGNIWaRmDtvQCcmjerFkrVb+M9p8/l42+6oI4lkjNCKwN1LgSuAZ4F3gIfcfZmZXWdmQ4LF/gAUAA+b2RIzm1PBy4kcsjOOP5K7xvblw027GD5lHms+2Rl1JJGMYHXtaIrCwkIvKiqKOobUYW+u3cIFdy8iy+DuC/pyXNumUUcSCZ2ZLXb3wvLm6RAKiZ0T2jfj4fEDyMtOMHLaAuat+iTqSCKRUhFILH2tdQGPXDqQI5vlM/bORTzz9oaoI4lERkUgsXV403weumQAx7VtwmUPvM6DCz+MOpJIJFQEEmvNGubywMX9+cbRrfmfR5dyywvv6yxkiR0VgcReg9wE035YyNm92vLHv7/H/z6+nP37VQYSHxlxZrFI1HISWdw04gRaNMpl+qtr2LSzhD+OOIHcbH1WkvpPRSASyMoyrjm9Gy0LcrnxmRVs+WwvU8f0pmGufk2kftPHHZEUZsZl3zyK3w3ryavvFzP69oVs3lkSdSyRUKkIRMoxsm8Hpozpw/IN2xhx23w+2vJZ1JFEQqMiEKnA93sczr0X9uXjrbsZPmUeKzdujzqSSChUBCKV6N+lJTMu6c/efc45U+fzxoebo44kUuNUBCJV6HFkUx65dABN8nMYfftCXn6vOOpIIjVKRSCSho4tGzFr/AA6tWrExfcs4rElX7m0hkidpSIQSVObJvnMGNefXh2a8+OZS7h77pqoI4nUCBWBSDU0bZDDvRf25eRuh3Ht48v5099XaEgKqfNUBCLVlJ+TYMoPenNuYTsmvrCSq2e/zT4NSSF1mE6ZFDkI2Yksfj/8eFoW5DHlpVVs3lnCzSNPJC87EXU0kWpTEYgcJDPjF6ccS8tGufzmyXf49I7X+NaxbSjIz6ZJfjaN87MpyMsJfmbTJD+HgvxsElkWdXSRL1ERiByii7/ehRaNcvnV7Ld57YNNVS7fMDdB4/xsGufnUJCXHdzPpvGB0gjmJad9cb8gZbn8nCzMVChSM1QEIjVgWO92nN2rLbv37mf77r1s213Kjj2lbN+9l+27S9mxu5Rtu/cG05LTD9zftruUj7Z8llxuTym7SvZV+X7ZWfZFOQQF8tVyydHWiaRFRSBSQ8yMBrkJGuQmaNPk4F+ndN/+lML4cmlsL1MuB+5v31PK+i272bFn++fPS2cHtrZOBFQEIhknO5FFs4a5NGuYe9Cv4e7lbp3s2H1gK6T8rZPttbh1UpCXTX5OFrnZyVtedoK87CxyE1lkaUulVqkIROqhsLZOUr/uKm/rZMee5FddB7N1kionYeQmviiIL8oiKI5EFnk5ieTPcudlkZtIfHl6cD8vu8zrJrLKWS5epaQiEJEK1fTWyfbUrZDdpewp3U9J6X72lO5L3t934HHy54F5JcG8PXuDn6X72frZ3mCZfV8sn/L8mjq3Iw6lpCIQkVB9aeukFt93337/UpHsSS2YffvZs3df+cVT2bw0Sil1Xk2X0vVnHcd5/TvWyGulUhGISL2UyPqigKJWnVJKLZ+ypXR826ah5FMRiIiELJNKqTwaa0hEJOZUBCIiMaciEBGJORWBiEjMqQhERGJORSAiEnMqAhGRmFMRiIjEXKhFYGanmNkKM1tpZleVMz/PzGYG8xeaWacw84iIyFeFVgRmlgAmA6cC3YFRZta9zGIXAZvd/Sjgz8Dvw8ojIiLlC3OLoC+w0t1Xu3sJMAMYWmaZocA9wf1ZwHdMV7gQEalVYY411BZYm/J4HdCvomXcvdTMtgItgU9SFzKzccC44OEOM1txkJlalX3tDKFc1aNc1Zep2ZSreg4lV4XDltaJQefcfRow7VBfx8yK3L2wBiLVKOWqHuWqvkzNplzVE1auML8aWg+0T3ncLphW7jJmlg00BT4NMZOIiJQRZhEsArqaWWczywVGAnPKLDMHOD+4fw7wgrvXzBUcREQkLaF9NRR8538F8CyQAO5092Vmdh1Q5O5zgOnAfWa2EthEsizCdMhfL4VEuapHuaovU7MpV/WEksv0AVxEJN50ZrGISMypCEREYq7eFEEaw1n8h5m9bmalZnZOmXnnm9n7we38ss+NMNc+M1sS3MruaA8710/MbLmZvWVm/zCzjinzolxfleWKcn2NN7OlwXu/mnoWvZn9MnjeCjP7fibkMrNOZvZZyvqaWpu5UpYbbmZuZoUp0yJbXxXlinp9mdlYMytOef+LU+Yd+u+ju9f5G8md0auALkAu8CbQvcwynYDjgXuBc1KmtwBWBz+bB/ebR50rmLcjwvX1LaBhcP9SYGaGrK9yc2XA+mqScn8I8Exwv3uwfB7QOXidRAbk6gS8HdX6CpZrDPwTWAAUZsL6qiRXpOsLGAvcUs5za+T3sb5sEVQ5nIW7f+DubwH7yzz3+8Bz7r7J3TcDzwGnZECuMKWT60V33xU8XEDyPBCIfn1VlCtM6eTalvKwEXDgKIyhwAx33+Pua4CVwetFnStM6QwvA3A9yfHFdqdMi3R9VZIrTOnmKk+N/D7WlyIobziLtrXw3LBfO9/MisxsgZmdVUOZDibXRcDTB/nc2soFEa8vM7vczFYBNwJXVue5EeQC6Gxmb5jZy2b29RrKlFYuM+sNtHf3J6v73IhyQYTrKzA8+Ep0lpkdOFm3RtZXnRhiIsY6uvt6M+sCvGBmS919VW0GMLMxQCHwjdp836pUkCvS9eXuk4HJZjYauIYvTpaMVAW5NgAd3P1TM+sDzDazHmW2IEJhZlnAn0h+3ZExqsgV2foKPA781d33mNklJAfr/HZNvXh92SJIZziLMJ4b6mu7+/rg52rgJaBXbeYys5OBq4Eh7r6nOs+NIFfk6yvFDODAFknk66u8XMFXL58G9xeT/I766FrK1Rg4DnjJzD4A+gNzgh2zUa6vCnNFvL5w909T/q3fAfRJ97lpCWPnR23fSG7ZrCa5c+nAzpYeFSx7N1/dWbyG5I6W5sH9FhmQqzmQF9xvBbxPOTu2wspF8o/oKqBrmemRrq9KckW9vrqm3D+T5NnzAD348s7P1dTczs9DydX6QA6SOynXR/HvPlj+Jb7YKRvp+qokV6TrCzgi5f7ZwILgfo38Ph7yf0Sm3IDTgPeCPxJXB9OuI/mpEeAkkt+f7SQ5sN2ylOdeSHKn1ErggkzIBQwElgb/KJYCF9VyrueBj4ElwW1OhqyvcnNlwPr6C7AsyPRi6i8yycGTU5cAAAGVSURBVK2XVcAK4NRMyAUMT5n+OnBmbeYqs+xLBH9wo15fFeWKen0BNwTv/2bw//HYlOce8u+jhpgQEYm5+rKPQEREDpKKQEQk5lQEIiIxpyIQEYk5FYGISMypCEREYk5FICIScyoCkWowszFm9lowJvxtZpYwsx1m9mczWxZcI6F1sOyV9sW1E2ZEnV2kIioCkTSZWTfgP4FB7n4isA/4AcnhnYvcvQfwMvDr4ClXAb3c/XhgfASRRdKi0UdF0vcdkoN9LTIzgAbARpLXkpgZLHM/8Lfg/lvAA2Y2G5hdu1FF0qctApH0GXCPu58Y3I5x92vLWe7AuC2nA5OB3iTLQx+8JCOpCETS9w/gHDNrA2BmLSx5zeQs4MD1pkcDrwZj27d39xeBXwBNgYIIMotUSZ9QRNLk7svN7Brg78Ef+r3A5SRHju0bzNtIcj9CArjfzJqS3JKY6O5bIoouUimNPipyiMxsh7vr077UWfpqSEQk5rRFICISc9oiEBGJORWBiEjMqQhERGJORSAiEnMqAhGRmPs/CmzE8yd69/kAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "eps_list = [0.1, 0.2, 0.3, 0.4, 0.5]\n",
    "score_list = list()\n",
    "\n",
    "for eps in eps_list:\n",
    "    pgd = ProjectedGradientDescent(classifier=art_classifier, norm=np.inf, eps=eps, eps_step=0.05, max_iter=20, \n",
    "                                   targeted=False, num_random_init=0, batch_size=128)\n",
    "    x_test_adv = pgd.generate(x_test)\n",
    "    score = model.score(x_test_adv, y_test)\n",
    "    score_list.append(score)\n",
    "\n",
    "plt.plot(eps_list, score_list)\n",
    "plt.xlabel('eps')\n",
    "plt.ylabel('Test Accuracy')\n",
    "plt.ylim((0, 1));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Targeted PGD attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "pgd = ProjectedGradientDescent(classifier=art_classifier, norm=np.inf, eps=0.5, eps_step=0.01, max_iter=50, \n",
    "                               targeted=True, num_random_init=3, batch_size=128)\n",
    "y_test_target = np.zeros((y_test.shape[0], 10))\n",
    "target_label = 7\n",
    "y_test_target[:, target_label] = 1\n",
    "x_test_adv = pgd.generate(x_test, y=y_test_target)\n",
    "score = model.score(x_test_adv, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Targeted Adversarial Test Score: 0.1900\n"
     ]
    }
   ],
   "source": [
    "score = model.score(x_test_adv, np.argmax(y_test_target, axis=1))\n",
    "print(\"Targeted Adversarial Test Score: %.4f\" % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAECCAYAAAD+eGJTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAWCklEQVR4nO3de3RV1Z0H8O8vyQ2PkPAGA4mABJzSdgkYX1VbqlZtOzPoaheMXavD2I64Rpyl1NZXO5U1nRZ8INPVqXRQWOKM6Dj4nKlWLVrfYoMioIhAEoZHSMAgCUFJcvObP3JpI+b89iUn95wT9/ezlovk/u45Z+fk5uu5d++zt6gqiMhfeXE3gIjixRAg8hxDgMhzDAEizzEEiDzHECDyXCwhICIXi8gWEdkmIjfG0QaLiNSKyEYRWS8iVQlozwoRaRCRTV0eGyYiz4rI1sy/QxPWvgUisjtzDteLyDdibF+5iDwvIu+KyDsick3m8UScQ6N9kZxDiXqcgIjkA3gfwNcA7ALwRwCXqeq7kTbEICK1ACpVdX/cbQEAEfkygEMA7lPVL2Qeuw1Ao6ouygTpUFW9IUHtWwDgkKreEUebuhKRUgClqvqmiBQDWAfgEgB/hwScQ6N9sxDBOYzjSuB0ANtUtVpVWwE8CGBmDO3oM1T1RQCNxzw8E8DKzNcr0fmiiUVA+xJDVetU9c3M180ANgMYi4ScQ6N9kYgjBMYC2Nnl+12I8AfOkgJ4RkTWicjcuBsTYLSq1mW+3gtgdJyNCXC1iGzIvF2I7e1KVyIyHsA0AGuRwHN4TPuACM4hPxjs3jmqOh3A1wHMy1zuJpZ2vqdL2vjvpQAmApgKoA7A4nibA4jIIAAPA7hWVZu61pJwDrtpXyTnMI4Q2A2gvMv3ZZnHEkNVd2f+bQDwKDrfwiRNfea95NH3lA0xt+cTVLVeVdOq2gHgbsR8DkUkhc4/sPtV9ZHMw4k5h921L6pzGEcI/BHAJBGZICKFAP4GwBMxtKNbIlKU+XAGIlIE4EIAm+ytYvEEgDmZr+cAeDzGtnzK0T+ujEsR4zkUEQGwHMBmVb2zSykR5zCofVGdw8h7BwAg09XxrwDyAaxQ1Z9H3ogAInISOv/vDwAFAFbF3T4ReQDADAAjANQDuAXAYwAeAnAigB0AZqlqLB/OBbRvBjovYxVALYAru7z/jrp95wB4CcBGAB2Zh29G5/vu2M+h0b7LEME5jCUEiCg5+MEgkecYAkSeYwgQeY4hQOQ5hgCR52INgQQPyQXA9oWV5PYluW1AtO2L+0og0b8IsH1hJbl9SW4bEGH74g4BIopZqMFCInIxgF+ic+TfPaq6yHp+ofTT/ij60/dtOIIU+mV9vNbSIrNeWNeS9b66I/0+2ZbW9GEU5g/s8gR7+9YhBWY9tTdc+451vOcvrI5h9vk/VtvHLUj1//M2eY29+/Mft6IBf/qyra0FqdQxP4/jb6G9KN+sS4dZRkHTEbOuhak/t6+9BamCY9rX8pHdvpHBv5/W5ka0f9TS7SvYftUaMpOD/BpdJgcRkSesyUH6owhnyPk9PSR2XvEls17+87VmHR1ps5w/fqK9fco+XTv+erhZL1v4qr1/F3GkUI5HfzZfdGao7YsffL2XWtIzesopZj2vtd2sN5xWYtbz7b9xjHqq2qy3nXSCWZdX3zbr+791VmBty8NLAmth3g5wchCiz4AwIdAXJgchIocevx3IVqarYy4A9MdAx7OJKGphrgSymhxEVZepaqWqVkb5IRYRZSdMCCR6chAiyk7YLsLjmhykf1m5ls+bH1ifeN8+83jpLdvMet7UKWa97twhZn3Mg1vNuo4dadY/GmN3oR2YlDLr4vhVlN6z3qx3HD5s7yAkOfXzZj2/8ZBjB3bvRnt17XG26JPq/9HuPRr9q3C9M9sX270jJdvsn2/k0tdCHT+MtboGTdrYu12EAKCqTwJ4Msw+iCheHDFI5DmGAJHnGAJEnmMIEHmOIUDkOYYAkeciXXegpKRMKyvnBdZrv2mPKCyxb8Jy9sPmj7T7+dP77HEK++cG36UFAPmtZhltA+1+5FF3hbzL0KFmod3+CTfF14+dBPkl9l2CUjzIrLfv3hPq+AUn2Ouhtu+tN+u1Pwv+/e68awk+3r2z2xcgrwSIPMcQIPIcQ4DIcwwBIs8xBIg8xxAg8hxDgMhz0Y4TkGFqzTbsul9d171j1l39rOnyUWa9pcye/mzg41Vm3TWbcVh7rrfvlz9cas95/Re31pj1926YYNYr5sc7W3Bf938L7N/fCa/ZA00Kn3a8/gzWfAK8EiDyHEOAyHMMASLPMQSIPMcQIPIcQ4DIcwwBIs/lfBmyrqRfIfJPDO6LlgOOpavHlZvl9h07zToc92MXvWWfDnWMA3DNe39ovN2Pr45Izmu1x3S4+vHtNXeBivn2+fHdtiX2ugPD37bnizhxgT1fRM0vHPM9PG2We4xXAkSeYwgQeY4hQOQ5hgCR5xgCRJ5jCBB5jiFA5LlEzScQlmv9+IG77cwrvTO38/5TslUvsvvpi3fY2x+cbP8tlWy1X3+udSf2znfMR7AkeHtrPoFQg4VEpBZAM4A0gHZVrQyzPyKKXm+MGPyqqu7vhf0QUQz4mQCR58KGgAJ4RkTWicjc3mgQEUUr7NuBc1R1t4iMAvCsiLynqi92fUImHOYCQH/YE3kSUfRCXQmo6u7Mvw0AHgVwejfPWaaqlapamYK96jARRa/HISAiRSJSfPRrABcC2NRbDSOiaIR5OzAawKMicnQ/q1T1d2Ea037eqWa94Ll1Zn3YRvt+7pF/2GUf36z2fflTJpv1nX85wqy71jXoKLbP4OdubTTr6a3VZj3XOvrZ/fwH7dOHycs/tPe/6b3jbdInWOMAAODwpWcEH/u54LkmehwCqloN4JSebk9EycAuQiLPMQSIPMcQIPIcQ4DIcwwBIs8xBIg8F+m6A+kRRWicGXzP9rAVr5nb77nevp96zG12P6qWlJh16WePaNy6aJpZdxnynj2OYcS/2z9/82x7voSGSnv/ELsfPDWu2d68dpBZH1BTaNZPWrXbrG89zSyH9sEV9nwBnbfCBHOt62CPogivoLzMrA98dG1gLU+D1/TglQCR5xgCRJ5jCBB5jiFA5DmGAJHnGAJEnmMIEHkuUesOdJxr98PnvfSWWXetHx83Hd5q1qsvWGHWJzz192a9ZKPdTz925Waznj5wwKyH9fSe9Wb9Jw1fNOurXjjbrE964LDdgNc32PU+Lq9//8Da6x8/iYMdH3Q7kIRXAkSeYwgQeY4hQOQ5hgCR5xgCRJ5jCBB5jiFA5LlEjRNw9fO77ucOy3X8fh/YmXlkuH1H+fbZvznuNnW17og9zuDH377crDdVFJv1escwi7I19s+363z7/BRNOGjWD+0YbNYnXWP//l3jECY8Zi+XOfmqN8x6X7ZW16BJGzlOgIg+jSFA5DmGAJHnGAJEnmMIEHmOIUDkOYYAkeciXXcgrJpf2PPGT7jZnre/4Wp73QLXzPHl/2Kva/CtzQ2O/Yfz7WfnmfXxI+wxH0Nf3WXW68+057VvGZ1v1ivm2+ffNe//qEv3mXXXOACXmkuWmfWLrpoaav8NV9mvr1F32a8f5/4dr9/+jcGv3/RTwWMsnFcCIrJCRBpEZFOXx4aJyLMisjXz71DXfogombJ5O3AvgIuPeexGAGtUdRKANZnviagPcoaAqr4IoPGYh2cCWJn5eiWAS3q5XUQUkZ5+MDhaVesyX+8FMLqX2kNEEQvdO6CddyAFfiIlInNFpEpEqtpwJOzhiKiX9TQE6kWkFAAy/wZ+LK6qy1S1UlUrU7BX/SWi6PU0BJ4AMCfz9RwAj/dOc4goas5xAiLyAIAZAEaIyC4AtwBYBOAhEfk+gB0AZmV1tKIB0FNOCSyXPWf30xe9f+znk5+0xTEfwKT/tO9nP+Flu5/dtf78PdX2vPhzp/23Yw+2vBa7n77waft+ez25ItTxhy+3xwG4jFxrr2vQcfc2ewd7Qh0e9zcPN+v5FRPMenpbjVlvmmS/QkaZVbdR/2aPM6hZGDwOI/2H4O2cIaCqlwWUgmcHIaI+g8OGiTzHECDyHEOAyHMMASLPMQSIPMcQIPJcotYdCMu1boBL4Yd2JrYOye26AjfU2/ez/89q+37y1mF2+yZel9t1G8JKz5hu1n+/akWo/U+vmm3WD26374jP9boXLvlTJpv19LvvB9a47gARBWIIEHmOIUDkOYYAkecYAkSeYwgQeY4hQOS5RK070HpRpVkvfLoqp8cf87I9/VnhvhZ7B3Y3tNPafePNetnz9vHl1bfDNSBmP1l+b073f/gtez6Bilvs+/Vd41ByPo5Auu3mD41XAkSeYwgQeY4hQOQ5hgCR5xgCRJ5jCBB5jiFA5LlEjRNwjQPQs+377Sc+9JFZ3z5rgH38N4LvxwaA927/nFkPq9+FtTndf641zLPnOzjve3Y/+owBrpUdbBXPX27WtcTev2scQPkzabO+54f2zz/mDnscgkv6nS2htg/CKwEizzEEiDzHECDyHEOAyHMMASLPMQSIPMcQIPJcpOMEjpQVYfsPgvti847Y90unmu166xB7DYXBjm7W9//582Zd0uHWaHin1R7HgNO/aNff2Bjq+GFVr7LHabxy7u1mfVR+UW8251PKRh4w6/rgKLM+aEOdWX/v2rFmfdxTrWY9rLyBA816x+HDPduv6wkiskJEGkRkU5fHFojIbhFZn/nvGz06OhHFLpu3A/cCuLibx5eo6tTMf0/2brOIKCrOEFDVFwE0RtAWIopBmA8GrxaRDZm3C/YibkSUWD0NgaUAJgKYCqAOwOKgJ4rIXBGpEpGqdItjok4iilyPQkBV61U1raodAO4GcLrx3GWqWqmqlflFuf10mIiOX49CQERKu3x7KYBNQc8lomRzjhMQkQcAzAAwQkR2AbgFwAwRmQpAAdQCuDKbg0kaSDUF507ZGrufM+/l9Wb9yDdPM+s7L8g36y7Sao9TOPmlvzXrr3xpqVnfPtu+UiqZfpZZ70jZ7Ru7utqsb751jFmvnrHCrAPxXum552Ow6+2OrSvm7zTrznUJnrH3337eqWa94Ll1Zr1mYfDro/VXwXM5OENAVS/r5uHlru2IqG/gsGEizzEEiDzHECDyHEOAyHMMASLPMQSIPBfpfAKFew9j3KLgvs6an043t+93hj2ve+lix7zuF9j9uIUH7UxsHWzPW9++x77f+9eN9jiG4hr7+E0Vrnn57fkOfrvud47tba75EH4w3h7H8PQee5yHy5W77P0Djvkacqxivr2ugktLacqsD3ZsLz1ctoFXAkSeYwgQeY4hQOQ5hgCR5xgCRJ5jCBB5jiFA5LlIxwkAADqC+7JTTfb98BpuOgCnExeEWz++5hd2P/bLV59h1g/OTpv1SVevPe42dfW1aX9l1qs32PPqu/rB83I8c1Tt6fGOA8i11hL79e8y/sevBdbqNXhqP14JEHmOIUDkOYYAkecYAkSeYwgQeY4hQOQ5hgCR5yIdJ9A+fCD2Xxo8t/rYW8P10++6yZ5vAOjhDddZmnBzcD9tNia91EsNCZB3vj1vfgXsustHX57ieMYrZvWItpn1/NGjzHq6vsFx/GQbudR+/bhe30V1wWNw0k8Ej/HglQCR5xgCRJ5jCBB5jiFA5DmGAJHnGAJEnmMIEHku0nECBftbMPye4L7Qjq9MM7fPe+Ets1620B5nkDfQXhcgt6MIPvuKNtWF2n5O7UVmva+PA3DZfrs9H8XkZXvNunx0JLC2tak1sOa8EhCRchF5XkTeFZF3ROSazOPDRORZEdma+Xeoa19ElDzZvB1oB3Cdqk4BcCaAeSIyBcCNANao6iQAazLfE1Ef4wwBVa1T1TczXzcD2AxgLICZAFZmnrYSwCW5aiQR5c5xfTAoIuMBTAOwFsBoVT36JnAvgNG92jIiikTWISAigwA8DOBaVW3qWlNVRcBqmCIyV0SqRKSqDcEfXBBRPLIKARFJoTMA7lfVRzIP14tIaaZeCqDbj25VdZmqVqpqZQr9eqPNRNSLsukdEADLAWxW1Tu7lJ4AMCfz9RwAj/d+84go17IZJ3A2gO8C2CgiRxeYvxnAIgAPicj3AewAMMu1o/TwIjTODO4LbR5vz7s+7oUsWmuovX6qWQ+77oDvqi8/MdT2e26vMOsD8Eao/Sfd8A12Pb21usf7VmOuBmcIqOrLAIL+Os/vYZuIKCE4bJjIcwwBIs8xBIg8xxAg8hxDgMhzDAEiz0U6n0C6H9B0UnD9pNUHzO3D3u/fOrhvzxiwbcmZZr1ifvDc8lFY/b3FjmcMsKuPfbbHAeQPGWzWh/yHve6AVH7BrB8aVxRY61jDdQeIKABDgMhzDAEizzEEiDzHECDyHEOAyHMMASLPRTpOIK8VKN4RXNd3t5vbb19s95NPvM7uJ4+7H91lzw/t9ecr5id7voOZq+eb9W3f+U1ELUmmvZdNMesjl9rjBNqLC836gP3G2gJtwWNkeCVA5DmGAJHnGAJEnmMIEHmOIUDkOYYAkecYAkSei3ScQMH+Fgy/J7gvtO5au598xFt9ez4Al1RLtyu5RcY1X8H22XY/f8Uqx3wH9/+DWZ/U/y2z3vHxx2Y96U54aItZTzu2z3/+TbN+aFbw+e9IBf//nlcCRJ5jCBB5jiFA5DmGAJHnGAJEnmMIEHmOIUDkOec4AREpB3AfgNEAFMAyVf2liCwAcAWAfZmn3qyqT4ZpTLqfo7Ef2/3oecXFZr2judms69lTzbq8st6suxSMP9GsH5xs/3wjQx0d2PMjexzG5JvsfmjMtsth5wu46Ef2+U+6mkVnmfUJN9rzBbjkDx1q1j+cGPz/dOtvK5vBQu0ArlPVN0WkGMA6EXk2U1uiqndksQ8iSihnCKhqHYC6zNfNIrIZwNhcN4yIonFcnwmIyHgA0wCszTx0tYhsEJEVImJfqxBRImUdAiIyCMDDAK5V1SYASwFMBDAVnVcK3S5EJyJzRaRKRKracKQXmkxEvSmrEBCRFDoD4H5VfQQAVLVeVdOq2gHgbgCnd7etqi5T1UpVrUzB8ckfEUXOGQIiIgCWA9isqnd2eby0y9MuBbCp95tHRLmWTe/A2QC+C2CjiBztI7sZwGUiMhWd3Ya1AK7MSQuJKKey6R14GYB0UzruMQHpYUVo+nrwPc9tg+x+8qLVa8162NkGwo4DcN2PH7fy/91n1rcstPvpJ/6XXR/32zaznvr9OrMet6bv2L+/IY9tMOt5wdP+AwBqf2aPIxj/T/Y4gvSHH9rHt06/8afFEYNEnmMIEHmOIUDkOYYAkecYAkSeYwgQeY4hQOS5SNcdgAAdBd0NOehUeDC41hvSM6ab9d1f6W/WW4eEG4lw8rIPzHp689ZQ+6++ze6HLjxgn9+K+a+GOr5rPoZcC3s/f8mq182667c/7qeOfn7H66+gvMyst+/cZdbH3BH8+9upLYE1XgkQeY4hQOQ5hgCR5xgCRJ5jCBB5jiFA5DmGAJHnRNW+h79XDyayD8COLg+NALA/sgYcP7YvnCS3L8ltA3q/feNUtdulKyINgU8dXKRKVStja4AD2xdOktuX5LYB0baPbweIPMcQIPJc3CGwLObju7B94SS5fUluGxBh+2L9TICI4hf3lQARxYwhQOQ5hgCR5xgCRJ5jCBB57v8BjB1QfdKPRL4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(x_test_adv[16, :].reshape((28, 28)))\n",
    "plt.clim(0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target Label: 7\n",
      "Predicted Label: 7\n"
     ]
    }
   ],
   "source": [
    "prediction = model.predict(x_test_adv[16:17, :])[0]\n",
    "print(\"Target Label: %i\" % target_label)\n",
    "print(\"Predicted Label: %i\" % prediction)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv36_TF210_keras231",
   "language": "python",
   "name": "venv36_tf210_keras231"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
